package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.abstarct.downloader.AbstractLoginDownLoader;
import com.spider.silence.constant.Constant;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.ChinaNewsForSiChuanTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.MD5Utils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;
import org.jsoup.nodes.Document;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Created by Linbo Hu on 2017/6/27.
 * 中国新闻网四川频道下载器
 */
public class ChinaNewsForSiChuanDownloader  extends AbstractCommonDownLoader implements Cloneable {

    private Logger logger =Logger.getLogger(ChinaNewsForSiChuanDownloader.class);

    private Map<String,File> files = null;

    public Map<String, File> getFiles() {
        return files;
    }

    private final String[] section = {
            "http://www.sc.chinanews.com/scxw/index.shtml", //四川新闻
            "http://www.sc.chinanews.com/glgj/index.shtml", //国内国际
            "http://www.sc.chinanews.com/zxjzzsc/index.shtml", //中新记者在四川
            "http://www.sc.chinanews.com/bwbd/index.shtml", //本网报道
            "http://www.sc.chinanews.com/rdzt/index.shtml", //热点专题
            "http://www.sc.chinanews.com/tpxw/index.shtml", //图片
            "http://www.sc.chinanews.com/szbd/index.shtml", //时政报道
            "http://www.sc.chinanews.com/szjj/index.shtml", //市州聚焦
            "http://www.sc.chinanews.com/hwmtksc/index.shtml", //海外媒体在四川
            "http://www.sc.chinanews.com/shms/index.shtml", //社会名声
            "http://www.sc.chinanews.com/cjbd/index.shtml", //财经
            "http://www.sc.chinanews.com/gatq/index.shtml", //港澳台侨
            "http://www.sc.chinanews.com/fcxw/index.shtml", //房地产
            "http://www.sc.chinanews.com/ylss/index.shtml", //娱乐时尚
            "http://www.sc.chinanews.com/whty/index.shtml", //文化体育
            "http://www.sc.chinanews.com/kjws/index.shtml", //科教卫生
            "http://www.sc.chinanews.com/shfw/index.shtml", //生活服务
            "http://www.sc.chinanews.com/lyxw/index.shtml"  //旅游
    };

    //中国新闻网四川频道网址
    private final String china_news_site = "http://www.sc.chinanews.com";



    //生成爬虫任务
    public void generateTask(){
        for(int i = 0; i < section.length; i++) {
            try {
                Page page = getHtml(section[i],true);
                if(page != null && page.getHtml() != null) {
                    Document document = page.getHtml().getDocument();
                    Elements elements = document.select("div[id=\"content\"] div[class=\"mt17\"] > ul[class=\"branch_list_ul paging\"] > li");
                    if(elements.size() > 0) {
                        int length = 50;
                        if(elements.size() < 50) {
                            length = elements.size();
                        }
                        for(int j = 0; j < length; j++) {
                            String url = china_news_site + elements.get(j).select("a").attr("href");
                            ChinaNewsForSiChuanTask task = new ChinaNewsForSiChuanTask(url);
                            CommonTaskManager.add("ChinaNewsForSiChuan", task);
                        }
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
                logger.error(e.getMessage(),e);
                continue;
            }
        }
    }

    public JSONObject crawlItem(String url) throws Exception {
        JSONObject item = new JSONObject();
        JSONObject data;
        boolean success = false;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("ChinaNewsForSiChuan.timeOut"))) {
            data = new JSONObject();
            data.put("url", url);
            nowTime = System.currentTimeMillis();
            try {
                Page page = this.getHtml(url,true);
                if(page != null && page.getStatusCode() == 200) {
                    Document document = page.getHtml().getDocument();
                    String title = document.title();
                    data.put("title", title);
                    String publish_time = document.select("div[id=\"content\"] div[class=\"f_l c_f_l\"] > div[class=\"c_info\"] > div[class=\"f_l\"]").get(0).text().substring(0, 17);
                    publish_time = (publish_time.replace(" ", " ")+":00").replace("年", "-").replace("月", "-").replace("日", "");
                    publish_time = DateUtils.strToFormatStr(publish_time);
                    data.put("publish_time", publish_time); //新闻发布时间
                    data.put("spider_time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));//爬取时间

                    Elements imgElements = document.select("div[id =\"txt_content\"] > p[style=\"text-align: center;\"] > img");
                    if(imgElements.size() > 0) {
                        List<String> img_urls = new ArrayList<>();
                        for(int i = 0; i < imgElements.size(); i++) {
                            String img = china_news_site + imgElements.get(i).attr("src");
                            img_urls.add(img);
                        }
                        data.put("img_urls", img_urls.toArray());
                    }

                    Elements contentElements = document.select("div[id =\"txt_content\"] > p");
                    if(contentElements.size() > 0) {
                        StringBuilder content = new StringBuilder();
                        for(int i = 0; i < contentElements.size(); i++) {
                            content.append(contentElements.get(i).text().trim());
                        }
                        data.put("content", content);
                    }
                    String newid = url.substring(url.lastIndexOf("/")+1, url.length()-5);
                    data.put("id", newid);
                    //设置html文件的命名格式
                    String fileName = "ChinaNewsForSiChuan" + "_" + DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime() + "_" + MD5Utils.getHash3(url.replaceAll("#.*", ""), "MD5");
                    String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                    String all_fileName = "中国新闻网四川频道_speeches_" + publishDate + "_" + fileName;
                    //抓取第一页的网页源码
                    File file = FileUtils.createTempFile(all_fileName + "_pageNo_1", document.html().toString());
                    files = new ConcurrentHashMap<>();
                    files.put(all_fileName + "_pageNo_1", file);
                    item.put("data", data);
                    success = true;
                    break;
                }
            } catch (Exception e) {
                //e.printStackTrace();
                Thread.sleep(1500);
                logger.info(e);
                System.err.println("爬虫过程出现异常, 尝试重新爬取");
                continue;
            }
        }
        if(success == true) {
            return item;
        } else {
            throw new Exception(url+"爬取失败");
        }
    }

    /**
     * 重写了getHtml方法, 使得能以不同的编码格式返回数据
     * @param url
     * @param isProxy
     * @return
     * @throws Exception
     */
    @Override
    public Page getHtml(String url, final boolean isProxy) throws Exception{
        downloader.setThread(10);
        request.setUrl(url);
        task=new Task() {
            @Override
            public String getUUID() {
                return UUID.randomUUID().toString();
            }
            @Override
            public Site getSite() {
                Site site = new Site();
                site.setCharset("gbk");
                site.setUserAgent(Constant.HTTP_USER_AGENT);
                site.setRetryTimes(Constant.HTTP_RETRY_TIMES);
                site.setCycleRetryTimes(Constant.HTTP_RETRY_TIMES);
                site.setTimeOut(Constant.HTTP_TIME_OUT);
                if(isProxy){
                    try {
                        downloader.setProxyProvider(SimpleProxyProvider.from(ProxyPool.proxyArray()));
                        downloader.setThread(10);
                    } catch (Exception e) {
                        System.out.printf("代理格式错误");
                    }
                }
                if(headers!=null){
                    for(Map.Entry<String, String> entry:headers.entrySet()){
                        site.addHeader(entry.getKey(),entry.getValue());
                    }
                }
                if(cookies!=null){
                    for (Cookie cookie : cookies) {
                        site.addCookie(cookie.getDomain(), cookie.getName(), cookie.getValue());
                    }
                }
                return site;
            }
        };
        return downloader.download(request,task);
    }

}
